const axios = require('axios');
const fs = require('fs');
const path = require('path');
const config = require('./config');

// 请求速率控制
class RateLimiter {
  constructor(maxRequests, interval) {
    this.maxRequests = maxRequests;
    this.interval = interval;
    this.queue = [];
    this.requestCount = 0;
    
    setInterval(() => {
      this.requestCount = 0;
      this.processQueue();
    }, this.interval);
  }
  
  async execute(requestFn) {
    return new Promise((resolve, reject) => {
      this.queue.push({ requestFn, resolve, reject });
      this.processQueue();
    });
  }
  
  processQueue() {
    while (this.queue.length > 0 && this.requestCount < this.maxRequests) {
      const { requestFn, resolve, reject } = this.queue.shift();
      this.requestCount++;
      
      requestFn()
        .then(resolve)
        .catch(reject);
    }
  }
}

// 检查robots.txt
async function checkRobotsTxt(baseUrl) {
  try {
    const robotsUrl = new URL('/robots.txt', baseUrl).toString();
    const response = await axios.get(robotsUrl);
    return response.data;
  } catch (error) {
    console.warn('无法获取robots.txt:', error.message);
    return null;
  }
}

// 计算文件哈希
function calculateFileHash(filePath) {
  return new Promise((resolve, reject) => {
    const hash = crypto.createHash('md5');
    const stream = fs.createReadStream(filePath);
    
    stream.on('error', reject);
    stream.on('data', chunk => hash.update(chunk));
    stream.on('end', () => resolve(hash.digest('hex')));
  });
}

// 日志记录
function log(message, level = 'info') {
  const timestamp = new Date().toISOString();
  const logMessage = `[${timestamp}] [${level.toUpperCase()}] ${message}\n`;
  
  // 控制台输出
  console[level](logMessage.trim());
  
  // 文件记录
  const logPath = path.join(__dirname, 'crawler.log');
  fs.appendFileSync(logPath, logMessage);
}

module.exports = {
  RateLimiter,
  checkRobotsTxt,
  calculateFileHash,
  log
};
